import { Button } from "@/src/components/ui/button"; import { DatasetCompareRunsTable } from "@/src/features/datasets/components/DatasetCompareRunsTable"; import { MultiSelectKeyValues } from "@/src/features/scores/components/multi-select-key-values"; import { api } from "@/src/utils/api"; import { FlaskConical, List } from "lucide-react"; import { useRouter } from "next/router"; import { useMemo, useState } from "react"; import { useQueryParams, withDefault, ArrayParam } from "use-query-params"; import { MarkdownJsonView } from "@/src/components/ui/MarkdownJsonView"; import { Dialog, DialogContent, DialogTrigger, } from "@/src/components/ui/dialog"; import { CreateExperimentsForm } from "@/src/features/experiments/components/CreateExperimentsForm"; import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess"; import { DatasetAnalytics } from "@/src/features/datasets/components/DatasetAnalytics"; import { TimeseriesChart } from "@/src/features/scores/components/TimeseriesChart"; import { isNumericDataType } from "@/src/features/scores/lib/helpers"; import { CompareViewAdapter } from "@/src/features/scores/adapters"; import { RESOURCE_METRICS, transformAggregatedRunMetricsToChartData, } from "@/src/features/dashboard/lib/score-analytics-utils"; import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture"; import Page from "@/src/components/layouts/page"; import { SubHeaderLabel } from "@/src/components/layouts/header"; import { SidePanel, SidePanelContent, SidePanelHeader, SidePanelTitle, } from "@/src/components/ui/side-panel"; import useLocalStorage from "@/src/components/useLocalStorage"; import { convertScoreColumnsToAnalyticsData, getScoreDataTypeIcon, scoreFilters, } from "@/src/features/scores/lib/scoreColumns"; export default function DatasetCompare() { const router = useRouter(); const capture = usePostHogClientCapture(); const projectId = router.query.projectId as string; const datasetId = router.query.datasetId as string; const [runState, setRunState] = useQueryParams({ runs: withDefault(ArrayParam, []), }); const [isCreateExperimentDialogOpen, setIsCreateExperimentDialogOpen] = useState(false); const [localRuns, setLocalRuns] = useState< Array<{ key: string; value: string }> >([]); const [selectedMetrics, setSelectedMetrics] = useLocalStorage( `${projectId}-dataset-compare-metrics`, RESOURCE_METRICS.map((metric) => metric.key), ); const runIds = runState.runs as undefined | string[]; const hasExperimentWriteAccess = useHasProjectAccess({ projectId, scope: "promptExperiments:CUD", }); const dataset = api.datasets.byId.useQuery({ datasetId, projectId, }); const runsData = api.datasets.baseRunDataByDatasetId.useQuery( { projectId, datasetId, }, { enabled: !!dataset.data, refetchOnWindowFocus: false, refetchOnMount: false, }, ); const utils = api.useUtils(); const runMetrics = api.datasets.runsByDatasetIdMetrics.useQuery( { projectId, datasetId, runIds: runIds ?? [], filter: [], }, { enabled: runIds && runIds.length > 1, }, ); const scoreKeysAndProps = api.scores.getScoreColumns.useQuery( { projectId: projectId, filter: runIds && runIds.length > 0 ? scoreFilters.forDatasetRunItems({ datasetRunIds: runIds, datasetId, }) : [], }, { enabled: runIds && runIds.length > 1, }, ); const scoreIdToName = useMemo(() => { return new Map( scoreKeysAndProps.data?.scoreColumns.map((obj) => [obj.key, obj.name]) ?? [], ); }, [scoreKeysAndProps.data]); const runAggregatedMetrics = useMemo(() => { return transformAggregatedRunMetricsToChartData( runMetrics.data?.runs.filter((run) => runIds?.includes(run.id)) ?? [], scoreIdToName, ); }, [runMetrics.data, runIds, scoreIdToName]); const { scoreAnalyticsOptions, scoreKeyToData } = useMemo( () => convertScoreColumnsToAnalyticsData(scoreKeysAndProps.data?.scoreColumns), [scoreKeysAndProps.data], ); const handleExperimentSettled = async (data?: { success: boolean; datasetId: string; runId: string; runName: string; }) => { setIsCreateExperimentDialogOpen(false); if (!data) return; void utils.datasets.baseRunDataByDatasetId.invalidate(); setLocalRuns((prev) => [...prev, { key: data.runId, value: data.runName }]); setRunState({ runs: [...(runIds ?? []), data.runId], }); }; const runs = useMemo(() => { const apiRuns = runsData.data?.map((run) => ({ key: run.id, value: run.name, })) ?? []; return [...apiRuns, ...localRuns]; }, [runsData.data, localRuns]); if (!runsData.data || !router.isReady || runs.length === 0) { return Loading...; } return ( } options={runs.map((run) => ({ key: run.key, value: run.value, disabled: runIds?.includes(run.key) && runIds.length === 1, }))} values={runs.filter((run) => runIds?.includes(run.key))} onValueChange={(values, changedValueId, selectedValueKeys) => { if (values.length === 0) return; if (changedValueId) { if (selectedValueKeys?.has(changedValueId)) { capture("dataset_run:compare_run_added"); setRunState({ runs: [...(runIds ?? []), changedValueId], }); setLocalRuns([]); } else { capture("dataset_run:compare_run_removed"); setRunState({ runs: runIds?.filter((id) => id !== changedValueId) ?? [], }); setLocalRuns([]); } } }} /> ), }} >
Compare Dataset Runs
{dataset.data?.description ?? "No description"}
{dataset.data?.metadata && (
)}
<> {Boolean(selectedMetrics.length) && Boolean(runAggregatedMetrics?.size) ? (
{selectedMetrics.map((key) => { const adapter = new CompareViewAdapter( runAggregatedMetrics, key, ); const { chartData, chartLabels } = adapter.toChartData(); const scoreData = scoreKeyToData.get(key); if (!scoreData) return (
metric.key === key, )?.label ?? key } type="numeric" maxFractionDigits={ RESOURCE_METRICS.find( (metric) => metric.key === key, )?.maxFractionDigits } />
); return (
); })}
) : ( {Boolean(runAggregatedMetrics?.size) ? "All charts hidden. Enable them in settings." : "Select more than one run to generate charts."} )}
); }